{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "!pip install -U  openai lancedb\n",
    "!pip install langchain-experimental langchain-openai langchain\n",
    "!pip install sentence-transformers"
   ],
   "metadata": {
    "id": "3lpJbF9cae32",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "e356123b-cda3-440c-d174-1c29c5455763"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (1.34.0)\n",
      "Requirement already satisfied: lancedb in /usr/local/lib/python3.10/dist-packages (0.8.2)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai) (0.27.0)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.7.3)\n",
      "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.4)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai) (4.12.2)\n",
      "Requirement already satisfied: deprecation in /usr/local/lib/python3.10/dist-packages (from lancedb) (2.1.0)\n",
      "Requirement already satisfied: pylance==0.12.1 in /usr/local/lib/python3.10/dist-packages (from lancedb) (0.12.1)\n",
      "Requirement already satisfied: ratelimiter~=1.0 in /usr/local/lib/python3.10/dist-packages (from lancedb) (1.2.0.post0)\n",
      "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from lancedb) (2.31.0)\n",
      "Requirement already satisfied: retry>=0.9.2 in /usr/local/lib/python3.10/dist-packages (from lancedb) (0.9.2)\n",
      "Requirement already satisfied: attrs>=21.3.0 in /usr/local/lib/python3.10/dist-packages (from lancedb) (23.2.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from lancedb) (24.1)\n",
      "Requirement already satisfied: cachetools in /usr/local/lib/python3.10/dist-packages (from lancedb) (5.3.3)\n",
      "Requirement already satisfied: overrides>=0.7 in /usr/local/lib/python3.10/dist-packages (from lancedb) (7.7.0)\n",
      "Requirement already satisfied: pyarrow<15.0.1,>=12 in /usr/local/lib/python3.10/dist-packages (from pylance==0.12.1->lancedb) (14.0.2)\n",
      "Requirement already satisfied: numpy>=1.22 in /usr/local/lib/python3.10/dist-packages (from pylance==0.12.1->lancedb) (1.25.2)\n",
      "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.1)\n",
      "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2024.6.2)\n",
      "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.18.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->lancedb) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->lancedb) (2.0.7)\n",
      "Requirement already satisfied: decorator>=3.4.2 in /usr/local/lib/python3.10/dist-packages (from retry>=0.9.2->lancedb) (4.4.2)\n",
      "Requirement already satisfied: py<2.0.0,>=1.4.26 in /usr/local/lib/python3.10/dist-packages (from retry>=0.9.2->lancedb) (1.11.0)\n",
      "Requirement already satisfied: langchain-experimental in /usr/local/lib/python3.10/dist-packages (0.0.61)\n",
      "Requirement already satisfied: langchain-openai in /usr/local/lib/python3.10/dist-packages (0.1.8)\n",
      "Requirement already satisfied: langchain in /usr/local/lib/python3.10/dist-packages (0.2.5)\n",
      "Requirement already satisfied: langchain-community<0.3.0,>=0.2.5 in /usr/local/lib/python3.10/dist-packages (from langchain-experimental) (0.2.5)\n",
      "Requirement already satisfied: langchain-core<0.3.0,>=0.2.7 in /usr/local/lib/python3.10/dist-packages (from langchain-experimental) (0.2.7)\n",
      "Requirement already satisfied: openai<2.0.0,>=1.26.0 in /usr/local/lib/python3.10/dist-packages (from langchain-openai) (1.34.0)\n",
      "Requirement already satisfied: tiktoken<1,>=0.7 in /usr/local/lib/python3.10/dist-packages (from langchain-openai) (0.7.0)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.30)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.9.5)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
      "Requirement already satisfied: langchain-text-splitters<0.3.0,>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.2.1)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.1.77)\n",
      "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.25.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.7.3)\n",
      "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.3.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain-community<0.3.0,>=0.2.5->langchain-experimental) (0.6.7)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.7->langchain-experimental) (1.33)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.7->langchain-experimental) (24.1)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /usr/local/lib/python3.10/dist-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.5)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (1.7.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (0.27.0)\n",
      "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (4.66.4)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.26.0->langchain-openai) (4.12.2)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.18.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2024.6.2)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.3)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<1,>=0.7->langchain-openai) (2024.5.15)\n",
      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.26.0->langchain-openai) (1.2.1)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community<0.3.0,>=0.2.5->langchain-experimental) (3.21.3)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community<0.3.0,>=0.2.5->langchain-experimental) (0.9.0)\n",
      "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.26.0->langchain-openai) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.26.0->langchain-openai) (0.14.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.3.0,>=0.2.7->langchain-experimental) (3.0.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community<0.3.0,>=0.2.5->langchain-experimental) (1.0.0)\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# load the data using pandas\n",
    "import pandas as pd\n",
    "import os\n",
    "import time\n",
    "import lancedb\n",
    "from lancedb.embeddings import EmbeddingFunctionRegistry\n",
    "import getpass\n",
    "import pandas as pd\n",
    "from lancedb.pydantic import LanceModel, Vector\n",
    "\n",
    "df = pd.read_csv(\"/content/hs_code_india_6digi_best.csv\")\n",
    "\n",
    "\n",
    "# Set the OPENAI_API_KEY environment variable\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-proj-\""
   ],
   "metadata": {
    "id": "ViKaBfBUaoqr"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "df.head()"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "QXSdP-XXdKZ-",
    "outputId": "59f14d98-fb4d-4a9d-945d-c91d8aa93c17"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   HSCode                                Commodity Year_2022_2023  \\\n",
       "0  100111                        DURUM WHEAT: SEED           4.91   \n",
       "1  100119              OTHER DURUM WHEAT EXCL SEED          29.41   \n",
       "2  100191  WHEAT AND MESLIN SEED EXCL. DURUM WHEAT         107.73   \n",
       "3  100199                   OTHER WHEAT AND MESLIN      46,941.02   \n",
       "4  100210                                RYE -SEED           3.16   \n",
       "\n",
       "   Share_Percentage_22_23 Year_2023_2024  Share_Percentage_23_24  \n",
       "0                   0.000              0                  0.0000  \n",
       "1                   0.000              0                  0.0000  \n",
       "2                   0.000           0.12                  0.0000  \n",
       "3                   0.013          94.27                  0.0003  \n",
       "4                   0.000              0                  0.0000  "
      ],
      "text/html": [
       "\n",
       "  <div id=\"df-f336d892-cffa-4f73-9091-40249af89129\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HSCode</th>\n",
       "      <th>Commodity</th>\n",
       "      <th>Year_2022_2023</th>\n",
       "      <th>Share_Percentage_22_23</th>\n",
       "      <th>Year_2023_2024</th>\n",
       "      <th>Share_Percentage_23_24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100111</td>\n",
       "      <td>DURUM WHEAT: SEED</td>\n",
       "      <td>4.91</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100119</td>\n",
       "      <td>OTHER DURUM WHEAT EXCL SEED</td>\n",
       "      <td>29.41</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100191</td>\n",
       "      <td>WHEAT AND MESLIN SEED EXCL. DURUM WHEAT</td>\n",
       "      <td>107.73</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100199</td>\n",
       "      <td>OTHER WHEAT AND MESLIN</td>\n",
       "      <td>46,941.02</td>\n",
       "      <td>0.013</td>\n",
       "      <td>94.27</td>\n",
       "      <td>0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100210</td>\n",
       "      <td>RYE -SEED</td>\n",
       "      <td>3.16</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f336d892-cffa-4f73-9091-40249af89129')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-f336d892-cffa-4f73-9091-40249af89129 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-f336d892-cffa-4f73-9091-40249af89129');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-1ac36d41-9293-4311-b33d-be7b1d87b5f9\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1ac36d41-9293-4311-b33d-be7b1d87b5f9')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-1ac36d41-9293-4311-b33d-be7b1d87b5f9 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "df",
       "summary": "{\n  \"name\": \"df\",\n  \"rows\": 320,\n  \"fields\": [\n    {\n      \"column\": \"HSCode\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 81570,\n        \"min\": 1,\n        \"max\": 310590,\n        \"num_unique_values\": 313,\n        \"samples\": [\n          22,\n          100490,\n          170113\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Commodity\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 309,\n        \"samples\": [\n          \"PLUMS AND SLOES , FRESH\",\n          \"OTHER OATS EXCL SEED\",\n          \"ESSENTIAL OILS AND RESINOIDS; PERFUMERY, COSMETIC OR TOILET PREPARATIONS.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Year_2022_2023\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 311,\n        \"samples\": [\n          \"101.38\",\n          \"515.79\",\n          \"2,453,055.10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Share_Percentage_22_23\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 65190.13279270224,\n        \"min\": 0.0,\n        \"max\": 1164333.35,\n        \"num_unique_values\": 173,\n        \"samples\": [\n          0.0047,\n          20.0428,\n          0.1663\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Year_2023_2024\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 282,\n        \"samples\": [\n          \"612,302.16\",\n          \"94,426.83\",\n          \"38,651.73\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Share_Percentage_23_24\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 76269.05263557596,\n        \"min\": 0.0,\n        \"max\": 1355788.69,\n        \"num_unique_values\": 172,\n        \"samples\": [\n          1.5327,\n          0.0168,\n          0.0033\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
      }
     },
     "metadata": {},
     "execution_count": 4
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Linear Combination Reranker\n",
    "This is the default re-ranker used by LanceDB. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search."
   ],
   "metadata": {
    "id": "nRYDIEKferCg"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "!pip install tantivy"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nJ8o5REjDuTg",
    "outputId": "4294133d-4cac-4cbd-ef6b-5aaf27497fbb"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Requirement already satisfied: tantivy in /usr/local/lib/python3.10/dist-packages (0.22.0)\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "import lancedb\n",
    "from lancedb.embeddings import EmbeddingFunctionRegistry\n",
    "from lancedb.rerankers import LinearCombinationReranker\n",
    "\n",
    "db = lancedb.connect(\"/tmp/aa\")\n",
    "registry = EmbeddingFunctionRegistry.get_instance()\n",
    "\n",
    "func = registry.get(\"sentence-transformers\").create(device=\"cpu\")\n",
    "\n",
    "\n",
    "class Words(LanceModel):\n",
    "    HSCode: int = func.SourceField()\n",
    "    Year_2022_2023: str = func.SourceField()\n",
    "    Year_2023_2024: str = func.SourceField()\n",
    "    Commodity: str = func.SourceField()\n",
    "    vector: Vector(func.ndims()) = func.VectorField()\n",
    "\n",
    "\n",
    "table = db.create_table(\"eximhs\", schema=Words, mode=\"overwrite\")\n",
    "table.add(data=df)\n",
    "\n",
    "# Create an FTS index with replace=True to overwrite if it already exists\n",
    "# we are doing FTS on commodity\n",
    "table.create_fts_index(\"Commodity\")\n",
    "\n",
    "# use can other rerankers such as colbert,coher etc\n",
    "# you can  check out blog for more https://blog.lancedb.com/hybrid-search-and-custom-reranking-with-lancedb-4c10a6a3447e/\n",
    "reranker = LinearCombinationReranker(\n",
    "    weight=0.3\n",
    ")  # Weight = 0 means pure Text Search (BM-25), 1 means pure Semantic (Vector) Search\n",
    "\n",
    "# reranker = LinearCombinationReranker(weight=0.3) # Use 0.3 as the weight for vector search\n",
    "# results = table.search(\"rebel\", query_type=\"hybrid\").rerank(reranker=reranker).to_pandas()\n",
    "\n",
    "\n",
    "query = \"what is HS code of sugar ?\"\n",
    "# Perform a hybrid search\n",
    "lance_reranker_op = (\n",
    "    table.search(query, query_type=\"hybrid\")\n",
    "    .rerank(reranker=reranker)\n",
    "    .limit(10)\n",
    "    .to_pandas()\n",
    ")\n",
    "\n",
    "lance_reranker_op"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 536
    },
    "id": "EPCSDijBae6u",
    "outputId": "575b4393-3cd2-4b59-b0a0-c0bbdbca0c08"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   HSCode Year_2022_2023 Year_2023_2024  \\\n",
       "0  170114     493,967.01      38,651.73   \n",
       "1      17   2,725,578.38     206,998.80   \n",
       "2      17   2,725,578.38     206,998.80   \n",
       "3  170290      41,657.79       3,899.81   \n",
       "4  170220           9.14           0.42   \n",
       "5  170113      56,381.70       4,173.90   \n",
       "6  170410      11,547.24       1,150.48   \n",
       "7  170260          11.15           0.03   \n",
       "8  170219         252.04           2.67   \n",
       "9   81110         671.91         689.66   \n",
       "\n",
       "                                           Commodity  \\\n",
       "0                                  OTHER CANE SUGAR:   \n",
       "1                    SUGARS AND SUGAR CONFECTIONERY.   \n",
       "2                    SUGARS AND SUGAR CONFECTIONERY.   \n",
       "3  OTHER, INCLUDING INVERT SUGAR AND OTHER SUGAR ...   \n",
       "4                        MAPLE SUGAR AND MAPLE SYRUP   \n",
       "5  CANE SUGAR SPECIFIED IN SUBHEADING NOTE 2 TO T...   \n",
       "6                       CHEWING GUM W/N SUGAR COATED   \n",
       "7  OTHER FRUCTOSE AND FRUCTOSE SYRUP, CONTAINING ...   \n",
       "8                    OTHER LACTOSE AND LACTOSE SYRUP   \n",
       "9  STRAWBERS,UNCOOKD/COOKD BY STEMNG/BOLNG IN WAT...   \n",
       "\n",
       "                                              vector  _relevance_score  \n",
       "0  [-0.09651693, -0.054201603, -0.08507634, 0.044...          0.943882  \n",
       "1  [-0.044817165, -0.061849847, -0.005438532, 0.0...          0.871490  \n",
       "2  [-0.044817165, -0.061849847, -0.005438532, 0.0...          0.871490  \n",
       "3  [-0.06565988, -0.09819972, -0.029922914, 0.074...          0.860843  \n",
       "4  [-0.055785332, -0.052705675, -0.028172577, 0.0...          0.728385  \n",
       "5  [-0.075881355, 0.009683033, -0.027468659, -0.0...          0.560491  \n",
       "6  [-0.082156956, -0.010985073, -0.010356254, 0.0...          0.532612  \n",
       "7  [-0.011077197, -0.06980431, -0.07643254, 0.056...          0.078796  \n",
       "8  [0.021483433, -0.12754624, -0.0002809844, 0.04...          0.049030  \n",
       "9  [-0.041225344, -0.051881045, 0.008875609, 0.04...          0.034394  "
      ],
      "text/html": [
       "\n",
       "  <div id=\"df-466b04f9-43d1-4d7f-9f2e-9591fa003e98\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HSCode</th>\n",
       "      <th>Year_2022_2023</th>\n",
       "      <th>Year_2023_2024</th>\n",
       "      <th>Commodity</th>\n",
       "      <th>vector</th>\n",
       "      <th>_relevance_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>170114</td>\n",
       "      <td>493,967.01</td>\n",
       "      <td>38,651.73</td>\n",
       "      <td>OTHER CANE SUGAR:</td>\n",
       "      <td>[-0.09651693, -0.054201603, -0.08507634, 0.044...</td>\n",
       "      <td>0.943882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17</td>\n",
       "      <td>2,725,578.38</td>\n",
       "      <td>206,998.80</td>\n",
       "      <td>SUGARS AND SUGAR CONFECTIONERY.</td>\n",
       "      <td>[-0.044817165, -0.061849847, -0.005438532, 0.0...</td>\n",
       "      <td>0.871490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>2,725,578.38</td>\n",
       "      <td>206,998.80</td>\n",
       "      <td>SUGARS AND SUGAR CONFECTIONERY.</td>\n",
       "      <td>[-0.044817165, -0.061849847, -0.005438532, 0.0...</td>\n",
       "      <td>0.871490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>170290</td>\n",
       "      <td>41,657.79</td>\n",
       "      <td>3,899.81</td>\n",
       "      <td>OTHER, INCLUDING INVERT SUGAR AND OTHER SUGAR ...</td>\n",
       "      <td>[-0.06565988, -0.09819972, -0.029922914, 0.074...</td>\n",
       "      <td>0.860843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>170220</td>\n",
       "      <td>9.14</td>\n",
       "      <td>0.42</td>\n",
       "      <td>MAPLE SUGAR AND MAPLE SYRUP</td>\n",
       "      <td>[-0.055785332, -0.052705675, -0.028172577, 0.0...</td>\n",
       "      <td>0.728385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>170113</td>\n",
       "      <td>56,381.70</td>\n",
       "      <td>4,173.90</td>\n",
       "      <td>CANE SUGAR SPECIFIED IN SUBHEADING NOTE 2 TO T...</td>\n",
       "      <td>[-0.075881355, 0.009683033, -0.027468659, -0.0...</td>\n",
       "      <td>0.560491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>170410</td>\n",
       "      <td>11,547.24</td>\n",
       "      <td>1,150.48</td>\n",
       "      <td>CHEWING GUM W/N SUGAR COATED</td>\n",
       "      <td>[-0.082156956, -0.010985073, -0.010356254, 0.0...</td>\n",
       "      <td>0.532612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>170260</td>\n",
       "      <td>11.15</td>\n",
       "      <td>0.03</td>\n",
       "      <td>OTHER FRUCTOSE AND FRUCTOSE SYRUP, CONTAINING ...</td>\n",
       "      <td>[-0.011077197, -0.06980431, -0.07643254, 0.056...</td>\n",
       "      <td>0.078796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>170219</td>\n",
       "      <td>252.04</td>\n",
       "      <td>2.67</td>\n",
       "      <td>OTHER LACTOSE AND LACTOSE SYRUP</td>\n",
       "      <td>[0.021483433, -0.12754624, -0.0002809844, 0.04...</td>\n",
       "      <td>0.049030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>81110</td>\n",
       "      <td>671.91</td>\n",
       "      <td>689.66</td>\n",
       "      <td>STRAWBERS,UNCOOKD/COOKD BY STEMNG/BOLNG IN WAT...</td>\n",
       "      <td>[-0.041225344, -0.051881045, 0.008875609, 0.04...</td>\n",
       "      <td>0.034394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-466b04f9-43d1-4d7f-9f2e-9591fa003e98')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-466b04f9-43d1-4d7f-9f2e-9591fa003e98 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-466b04f9-43d1-4d7f-9f2e-9591fa003e98');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-fe77264c-c4e2-4cd3-b55f-a013ca43080e\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fe77264c-c4e2-4cd3-b55f-a013ca43080e')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-fe77264c-c4e2-4cd3-b55f-a013ca43080e button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "\n",
       "  <div id=\"id_1c70c8b1-d0c6-41c9-82d7-14a2294c2ec8\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('lance_reranker_op')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_1c70c8b1-d0c6-41c9-82d7-14a2294c2ec8 button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('lance_reranker_op');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "lance_reranker_op",
       "summary": "{\n  \"name\": \"lance_reranker_op\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"HSCode\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 72600,\n        \"min\": 17,\n        \"max\": 170410,\n        \"num_unique_values\": 9,\n        \"samples\": [\n          170219,\n          17,\n          170410\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Year_2022_2023\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          \"252.04\",\n          \"2,725,578.38\",\n          \"11,547.24\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Year_2023_2024\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          \"2.67\",\n          \"206,998.80\",\n          \"1,150.48\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Commodity\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          \"OTHER LACTOSE AND LACTOSE SYRUP\",\n          \"SUGARS AND SUGAR CONFECTIONERY.\",\n          \"CHEWING GUM W/N SUGAR COATED\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"vector\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"_relevance_score\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          0.049030207097530365,\n          0.8714902997016907,\n          0.5326119661331177\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
      }
     },
     "metadata": {},
     "execution_count": 7
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# As we can see we are getting the ouput from our vectordb"
   ],
   "metadata": {
    "id": "fqpx0qmoD5Y2"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# now we are removing both columsn bcz its not importatn fro next steps\n",
    "df = lance_reranker_op.drop(columns=[\"_relevance_score\", \"vector\"])\n",
    "\n",
    "# The cleaned DataFrame\n",
    "result_from_lancedb = df"
   ],
   "metadata": {
    "id": "Uy_aPBlpg0AA"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "result_from_lancedb"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 363
    },
    "id": "y01VxN-2g97k",
    "outputId": "c5e5b43c-73ba-4889-b04e-13e3146b301c"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   HSCode Year_2022_2023 Year_2023_2024  \\\n",
       "0  170114     493,967.01      38,651.73   \n",
       "1      17   2,725,578.38     206,998.80   \n",
       "2      17   2,725,578.38     206,998.80   \n",
       "3  170290      41,657.79       3,899.81   \n",
       "4  170220           9.14           0.42   \n",
       "5  170113      56,381.70       4,173.90   \n",
       "6  170410      11,547.24       1,150.48   \n",
       "7  170260          11.15           0.03   \n",
       "8  170219         252.04           2.67   \n",
       "9   81110         671.91         689.66   \n",
       "\n",
       "                                           Commodity  \n",
       "0                                  OTHER CANE SUGAR:  \n",
       "1                    SUGARS AND SUGAR CONFECTIONERY.  \n",
       "2                    SUGARS AND SUGAR CONFECTIONERY.  \n",
       "3  OTHER, INCLUDING INVERT SUGAR AND OTHER SUGAR ...  \n",
       "4                        MAPLE SUGAR AND MAPLE SYRUP  \n",
       "5  CANE SUGAR SPECIFIED IN SUBHEADING NOTE 2 TO T...  \n",
       "6                       CHEWING GUM W/N SUGAR COATED  \n",
       "7  OTHER FRUCTOSE AND FRUCTOSE SYRUP, CONTAINING ...  \n",
       "8                    OTHER LACTOSE AND LACTOSE SYRUP  \n",
       "9  STRAWBERS,UNCOOKD/COOKD BY STEMNG/BOLNG IN WAT...  "
      ],
      "text/html": [
       "\n",
       "  <div id=\"df-fba8f2c7-6e38-4552-b823-10cafb62084c\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HSCode</th>\n",
       "      <th>Year_2022_2023</th>\n",
       "      <th>Year_2023_2024</th>\n",
       "      <th>Commodity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>170114</td>\n",
       "      <td>493,967.01</td>\n",
       "      <td>38,651.73</td>\n",
       "      <td>OTHER CANE SUGAR:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17</td>\n",
       "      <td>2,725,578.38</td>\n",
       "      <td>206,998.80</td>\n",
       "      <td>SUGARS AND SUGAR CONFECTIONERY.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>2,725,578.38</td>\n",
       "      <td>206,998.80</td>\n",
       "      <td>SUGARS AND SUGAR CONFECTIONERY.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>170290</td>\n",
       "      <td>41,657.79</td>\n",
       "      <td>3,899.81</td>\n",
       "      <td>OTHER, INCLUDING INVERT SUGAR AND OTHER SUGAR ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>170220</td>\n",
       "      <td>9.14</td>\n",
       "      <td>0.42</td>\n",
       "      <td>MAPLE SUGAR AND MAPLE SYRUP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>170113</td>\n",
       "      <td>56,381.70</td>\n",
       "      <td>4,173.90</td>\n",
       "      <td>CANE SUGAR SPECIFIED IN SUBHEADING NOTE 2 TO T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>170410</td>\n",
       "      <td>11,547.24</td>\n",
       "      <td>1,150.48</td>\n",
       "      <td>CHEWING GUM W/N SUGAR COATED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>170260</td>\n",
       "      <td>11.15</td>\n",
       "      <td>0.03</td>\n",
       "      <td>OTHER FRUCTOSE AND FRUCTOSE SYRUP, CONTAINING ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>170219</td>\n",
       "      <td>252.04</td>\n",
       "      <td>2.67</td>\n",
       "      <td>OTHER LACTOSE AND LACTOSE SYRUP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>81110</td>\n",
       "      <td>671.91</td>\n",
       "      <td>689.66</td>\n",
       "      <td>STRAWBERS,UNCOOKD/COOKD BY STEMNG/BOLNG IN WAT...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fba8f2c7-6e38-4552-b823-10cafb62084c')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-fba8f2c7-6e38-4552-b823-10cafb62084c button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-fba8f2c7-6e38-4552-b823-10cafb62084c');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-5ca2e959-0404-483a-9e58-684a8c0a3ec1\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5ca2e959-0404-483a-9e58-684a8c0a3ec1')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-5ca2e959-0404-483a-9e58-684a8c0a3ec1 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "\n",
       "  <div id=\"id_cb961694-a87f-43eb-a5ad-173fcd19f302\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_cb961694-a87f-43eb-a5ad-173fcd19f302 button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('df');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "df",
       "summary": "{\n  \"name\": \"df\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"HSCode\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 72600,\n        \"min\": 17,\n        \"max\": 170410,\n        \"num_unique_values\": 9,\n        \"samples\": [\n          170219,\n          17,\n          170410\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Year_2022_2023\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          \"252.04\",\n          \"2,725,578.38\",\n          \"11,547.24\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Year_2023_2024\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          \"2.67\",\n          \"206,998.80\",\n          \"1,150.48\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Commodity\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9,\n        \"samples\": [\n          \"OTHER LACTOSE AND LACTOSE SYRUP\",\n          \"SUGARS AND SUGAR CONFECTIONERY.\",\n          \"CHEWING GUM W/N SUGAR COATED\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
      }
     },
     "metadata": {},
     "execution_count": 9
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "lance_reranker_op = lance_reranker_op.copy()"
   ],
   "metadata": {
    "id": "HQsQVJ5dfwrA"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "import pandas as pd\n",
    "from langchain_openai import OpenAI\n",
    "from langchain.agents.agent_types import AgentType\n",
    "from langchain_experimental.agents.agent_toolkits import create_pandas_dataframe_agent\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "\n",
    "agent = create_pandas_dataframe_agent(\n",
    "    ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo-0613\"),\n",
    "    result_from_lancedb,\n",
    "    verbose=True,\n",
    "    agent_type=AgentType.OPENAI_FUNCTIONS,\n",
    "    handle_parsing_errors=True,\n",
    "    allow_dangerous_code=True,\n",
    ")\n",
    "\n",
    "\n",
    "agent.invoke(query)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KGmj_U9QazYD",
    "outputId": "ebdfc078-3f76-4f3c-9828-f4656e4587a2"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/langchain_experimental/agents/agent_toolkits/pandas/base.py:282: UserWarning: Received additional kwargs {'handle_parsing_errors': True} which are no longer supported.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3mThe HS code for sugar is 1701.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'input': 'what is HS code of sugar ?',\n",
       " 'output': 'The HS code for sugar is 1701.'}"
      ]
     },
     "metadata": {},
     "execution_count": 11
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "id": "e_-rup3yiQXO"
   },
   "execution_count": null,
   "outputs": []
  }
 ]
}